import aiohttp
import pymongo
from pymongo import ReturnDocument
from bs4 import BeautifulSoup
import requests
import random
from retry import retry

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36"}

USER_AGENT_LIST=[
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]

headers ={
    "User-Agent":random.choice(USER_AGENT_LIST)}
proxies = {'http': '127.0.0.1:15732',
           'https': '127.0.0.1:15732'}




def get_html(url):
    try:
        html = requests.get(url,headers=headers,proxies=get_proxies(),timeout=5)#超时设置为3秒
    except Exception as e:
        for i in range(2):  # 循环2次去请求网站
            html = requests.get(url,headers=headers,proxies=get_proxies(),timeout=10)
    soup = BeautifulSoup(html.text,"lxml")
    return soup

class MongoDBUtil:
    """
    MongoDB工具类
    """

    def __init__(self, ip="127.0.0.1", db_name="douban"):
        """构造函数"""
        self.client = pymongo.MongoClient(host=ip, port=27017) #username="root",password="123456"
        self.database = self.client[db_name]

    def __del__(self):
        """析构函数"""
        # print("__del__")
        self.client.close()

    def create_database(self, db_name):
        """创建数据库"""
        return self.client.get_database(db_name)

    def drop_database(self, db_name):
        """删除数据库"""
        return self.client.drop_database(db_name)

    def select_database(self, db_name):
        """使用数据库"""
        self.database = self.client[db_name]
        return self.database

    def get_database(self, db_name):
        """使用数据库"""
        # return self.client[db_name]
        return self.client.get_database(db_name)

    def list_database_names(self):
        """获取所有数据库列表"""
        return self.client.list_database_names()

    def create_collection(self, collect_name):
        """创建集合"""
        collect = self.database.get_collection(collect_name)
        if (collect is not None):
            print("collection %s already exists" % collect_name)
            return collect
        return self.database.create_collection(collect_name)

    def drop_collection(self, collect_name):
        """获取所有集合名称"""
        return self.database.drop_collection(collect_name)

    def get_collection(self, collect_name):
        """获取集合"""
        return self.database.get_collection(collect_name)

    def list_collection_names(self):
        """获取所有集合名称"""
        return self.database.list_collection_names()

    def insert(self, collect_name, documents):
        """插入单条或多条数据"""
        return self.database.get_collection(collect_name).insert(documents)

    def insert_one(self, collect_name, document):
        """插入一条数据"""
        return self.database.get_collection(collect_name).insert_one(document)

    def insert_many(self, collect_name, documents):
        """插入多条数据"""
        return self.database.get_collection(collect_name).insert_many(documents)

    def delete_one(self, collect_name, filter, collation=None, hint=None, session=None):
        """删除一条记录"""
        return self.database.get_collection(collect_name).delete_one(filter, collation, hint, session)

    def delete_many(self, collect_name, filter, collation=None, hint=None, session=None):
        """删除所有记录"""
        return self.database.get_collection(collect_name).delete_many(filter, collation, hint, session)

    def find_one_and_delete(self, collect_name, filter, projection=None, sort=None, hint=None, session=None, **kwargs):
        """查询并删除一条记录"""
        return self.database.get_collection(collect_name).find_one_and_delete(filter, projection, sort, hint, session,
                                                                              **kwargs)

    def count_documents(self, collect_name, filter, session=None, **kwargs):
        """查询文档数目"""
        return self.database.get_collection(collect_name).count_documents(filter, session, **kwargs)

    def find_one(self, collect_name, filter=None, *args, **kwargs):
        """查询一条记录"""
        return self.database.get_collection(collect_name).find_one(filter, *args, **kwargs)

    def find(self, collect_name, *args, **kwargs):
        """查询所有记录"""
        return self.database.get_collection(collect_name).find(*args, **kwargs)

    def update(self, collect_name, spec, document, upsert=False, manipulate=False,
               multi=False, check_keys=True, **kwargs):
        """更新所有记录"""
        return self.database.get_collection(collect_name).update(spec, document,
                                                                 upsert, manipulate, multi, check_keys, **kwargs)

    def update_one(self, collect_name, filter, update, upsert=False, bypass_document_validation=False,
                   collation=None, array_filters=None, hint=None, session=None):
        """更新一条记录"""
        return self.database.get_collection(collect_name).update_one(filter, update,
                                                                     upsert, bypass_document_validation, collation,
                                                                     array_filters, hint, session)

    def update_many(self, collect_name, filter, update, upsert=False, array_filters=None,
                    bypass_document_validation=False, collation=None, hint=None, session=None):
        """更新所有记录"""
        return self.database.get_collection(collect_name).update_many(filter, update,
                                                                      upsert, array_filters, bypass_document_validation,
                                                                      collation, hint, session)

    def find_one_and_update(self,collect_name,query,update,sort,projection=None,):
        return  self.database.get_collection(collect_name).find_one_and_update(query,{'$inc': update},return_document=ReturnDocument.AFTER,sort=sort,projection=projection)
    # def find_one_and_update(self, collect_name, filter, update, projection=None, sort=None, upsert=False,
    #                         return_document=False, array_filters=None, hint=None, session=None, **kwargs):
    #     """查询并更新一条记录"""
    #     return self.database.get_collection(collect_name).find_one_and_update(filter, {"$inc":update}, projection,  
    #                                                                           sort, upsert, return_document,
    #                                                                           array_filters, hint, session, **kwargs)

    def add_status_field(self):
        self.database.update_many({}, {'$set': {'status': '0'}})
class mongo_manager():
    def __init__(self, collect_name,db = 'douban'):
        self.client=pymongo.MongoClient(host="127.0.0.1", port=27017)
        # username="root",password="123456"
        self.db=self.client[db]
        self.collect=self.db[collect_name]

    def find_one_and_update(self,query,update,sort,projection=None,):
        return  self.collect.find_one_and_update(query,{'$set': update},return_document=ReturnDocument.AFTER,sort=sort,projection=projection)


    def getCollect(self,collect_name:str)->pymongo.collection.Collection :
        """
        创建集合
        :return:
        """
        if bool(collect_name):
            return self.db[collect_name]
        return  self.collect


    # def findById(self,id:str):
    #     myquery = {"_id": ObjectId(id)}
    #     return self.collect.find_one(myquery)


    def findAll(self,data=None):
        if bool(data):
            return self.collect.find(data)
        return  self.collect.find()

    def findOne(self,data=None):
        if bool(data):
            return self.collect.find_one(data)
        return  self.collect.find_one()

    def insertOne(self,data:dict):
        return self.collect.insert_one(data)

    def insertMany(self,datas:list):
        return self.collect.insert_many(datas)

    def updateOne(self,query,new_value):
        newvalues = {"$set": new_value}
        return self.collect.update_one(query, newvalues)

    def updateMany(self, query, new_value):
        newvalues = {"$set": new_value}
        return self.collect.update_many(query, newvalues)

    # def deleteById(self,id):
    #     return self.collect.remove({"_id", ObjectId(id)})

    def  deleteOne(self,myquery):
        return  self.collect.delete_one(myquery)

    def deleteMany(self,myquery):
        return  self.collect.delete_many(myquery)

    def close(self):
        self.client.close()
